﻿using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using MIL.Html;
using System.Net;
using System.IO;
using Common;
using System.Text.RegularExpressions;
using System.Threading;

namespace Common
{

	public class Crawler
	{
	/*	/// <summary>
		/// Locked int of object
		/// </summary>
		private int ExpectedWork { set { lock (_expectedLock) { _expectedWork = value; };} get { lock (_expectedLock) { return _expectedWork; }; } }
		/// <summary>
		/// Locked int of object
		/// </summary>
		private int DoneWork { set { lock (_doneLock) { _doneWork = value; };} get { lock (_doneLock) { return _doneWork; }; } }
		private int _expectedWork, _doneWork;
		private object _expectedLock, _doneLock;*/

        //public List<string> visitedLinks = new List<string>();
        public List<string> resultLinks = new List<string>();
        public object lockVisitedLinks = new object();
        public object lockResultLinks = new object();

        public class CrawlArgs
        {
            public string Url;
            public int Depth;
            public bool AllowBadChars;

            public CrawlArgs(string url, int depth, bool allowBadChars)
            {
                Url = url;
                Depth = depth;
                AllowBadChars = allowBadChars;
            }
           
        }		
        public List<string> Crawl(string url, int depth, int MaxThreads, int MinThreads)
		{
            //JobManager.SetMaxThreads(MaxThreads, MinThreads);
            //visitedLinks = new List<string>();
            //resultLinks = new List<string>();
			return Crawl(url, depth, false);
		}

        public List<string> Crawl(string url, int depth, bool allowBadChars)
        {

            Crawl(new CrawlArgs(url, depth, allowBadChars) as object);
			
            return resultLinks;
        }

		public void Crawl(object crawlArgs)
		{
            CrawlArgs args = crawlArgs as CrawlArgs;
            string url = args.Url;
            int depth = args.Depth;
            bool allowBadChars = args.AllowBadChars;

			if (depth < 0) {  return; }
            
			Uri tmpUri, uri;
            try
            {
                uri = new Uri(url);
            }
            catch //invalid URI.
            {
                Console.WriteLine("Crawler : Invalid URL = " + url);
                return;
            }
            string baseAddress = uri.Scheme + "://" + uri.Host;
            string address = baseAddress;
			for (int i = 0; i < uri.Segments.Length; i++)
			{
				if (!uri.Segments[i].EndsWith("/")) break;
				address += uri.Segments[i];
			}           
			//Console.Write("{0} : ", url);
			//List<string> links = new List<string>();
            /*lock (lockVisitedLinks)
            {
                if (visitedLinks.Contains(url))
				{ return; }
                visitedLinks.Add(url);
            }
            */
			var gathered = GetAllLinks(url);
            string slash = "/", javascript = "javascript";
			//Console.WriteLine("{0} link(s) found", gathered.Length);
			foreach (string link in gathered)
			{
			
				
				if (link == string.Empty) continue;
				if (link.ToLower().StartsWith(javascript)) continue;
				
				string nLink = link;

				if (nLink.StartsWith("\"") && nLink.EndsWith("\"")) nLink = nLink.Substring(1, link.Length - 2);
				if (!nLink.ToLower().StartsWith("http"))
				{
					if (nLink.StartsWith(slash)) nLink = baseAddress + nLink;
					else nLink = address + nLink;
				}
//				nLink = nLink.Replace("//", "/");
                //lock(resultLinks)
				  //  if (links.Contains(nLink)) continue;
				if (!IsLink(nLink, allowBadChars)) continue;

			/*
				try
				{
					tmpUri = new Uri(nLink);
				}
				catch (Exception e)
				{
                    Console.WriteLine("Crawler Exception: {0}", e.Message);
					continue;
				}
                 */
				//if (!SameDomain(url, nLink)) continue;
                lock(lockResultLinks)
                    if (!resultLinks.Contains(nLink))
				        resultLinks.Add(nLink);
				//if (depth > 0) JobManager.AddJob(Crawl, new CrawlArgs(nLink, depth - 1,allowBadChars));				
			}
		}

		private static bool IsLink(string link, bool allowBadChars)
		{
            char[] badChars = { };//'#' };
			string[] badSuffixes = { ".jpg", ".bmp", ".jpeg", ".mp3", ".avi", ".wav" };
			if (!allowBadChars)
			{
				foreach (char c in badChars)
				{
					if (link.Contains(c)) return false;
				}
			}
			foreach (string bad in badSuffixes)
			{
				if (link.ToLower().EndsWith(bad)) return false;
			}
			return true;
		}/*
		private static bool IsLink(string link)
		{
			char[] badChars = { '#' };
			string[] badSuffixes = { ".jpg", ".bmp", ".jpeg", ".mp3", ".avi", ".wav"};
			//if (!link.StartsWith("http://")) return false;
			foreach (char c in badChars)
			{
				if (link.Contains(c)) return false;
			}
			foreach (string bad in badSuffixes)
			{
				if (link.EndsWith(bad)) return false;
			}
			return true;
		}*/
		private static bool SameDomain(string link1, string link2)
		{
			Uri uri1 = new Uri(link1), uri2 = new Uri(link2);
			return (uri1.Host == uri2.Host);
		}

        public static string[] GetAllLinksByRule(string url, Rule rule)
        {
            try
            {
                string page = HtmlDownloader.DownloadPage(new Uri(url));
                var nodes = HtmlDownloader.GetNodes(page);
                return GetAllLinksByRule(nodes, rule);
            }
            catch (Exception e)
            {
                Console.WriteLine("Exception: {0}", e.Message);
                return new string[] { };
            }
        }

        public static string[] GetAllLinksByRule(HtmlNodeCollection Nodes, Rule Rule)
        {
            HtmlNode node = GetNodeByRule(Nodes, Rule);
            return GetAllLinks((node as HtmlElement).Nodes).ToArray();
        }

		public static string[] GetAllLinks(string url)
		{
			try
			{
				List<string> links = new List<string>();
				string page = HtmlDownloader.DownloadPage(new Uri(url),2);
			    var nodes = HtmlDownloader.GetNodes(page);
				return GetAllLinks(nodes);
                /*
				page = Regex.Replace(page, "&amp;", "&");

				var matches = Regex.Matches(page, "<a.*?href=[\"?'?](?<link>[^( |\"|>)]*?)[\"?'?]( .*?>|>)", RegexOptions.IgnoreCase);
				foreach (Match m in matches)
				{
					var link = m.Groups["link"].Value;
					if (link != string.Empty && !link.StartsWith("#")) links.Add(link);
				}
				return links.ToArray();
                 * */
			}
			catch (Exception e)
			{
				Console.WriteLine("Exception: {0}", e.Message);
				return new string[] { };
			}
		}

       

		private static string[] GetAllLinks(HtmlNodeCollection nodes)
		{
			List<string> links = new List<string>();

			foreach (HtmlNode node in nodes)
			{
				if (node is HtmlElement)
				{					
					HtmlElement e = node as HtmlElement;
					if (e.Name.ToLower() == "a" || e.Name.ToLower() == "link")
					{
						foreach (HtmlAttribute att in e.Attributes)
						{
							if (att.Name.ToLower() == "href")
							{
								string link = att.Value;
                                if (link.StartsWith(@"//"))
                                    link = "http:" + link;
                                if (!(links.Contains(link))) links.Add(link);
								break;
							}
						}
					}
					var gatheredLinks = GetAllLinks(e.Nodes);
					foreach (string link in gatheredLinks)
					{
						if (!(links.Contains(link))) links.Add(link);
					}
				}				
			}
			return links.ToArray();
		}
		public static HtmlNode GetNodeByRule(HtmlNodeCollection nodes, Rule rule)
		{
			foreach (HtmlNode childNode in nodes)
			{
				if (childNode is HtmlElement)
				{
					HtmlElement e = childNode as HtmlElement;
					if (e.Name.ToLower() == rule.PropertyName.ToLower())
						if (HasAttribValue(e.Attributes, rule.AttributeName, rule.AttributeValue))
							return childNode;
					HtmlNode ret = GetNodeByRule(e.Nodes, rule);
					if (null != ret) return ret;
				}
			}
			return null;
		}
		private static HtmlNode[] GetNodesByRule(HtmlNodeCollection nodes, Rule rule, bool partial)
		{
			List<HtmlNode> found = new List<HtmlNode>();
			foreach (HtmlNode childNode in nodes)
			{
				if (childNode is HtmlElement)
				{
					HtmlElement e = childNode as HtmlElement;
					if (IsElementRule(e, rule, partial) && !IsElementInExclusionList(e, rule.Exclusions))
						found.Add(e);

					var ret = GetNodesByRule(e.Nodes, rule, partial);
					if (ret != null) foreach (HtmlNode n in ret) found.Add(n);
				}

			}
			if (found.Count != 0) return found.ToArray();
			return null;
		}
		private static bool IsElementRule(HtmlElement element, Rule rule, bool partial)
		{
			if (element.Name.ToLower() == rule.PropertyName.ToLower()) //This is the property we're looking for.
			{
				bool hasAllAttributes = true;
				foreach (Common.Attribute a in rule.Attributes) //Check if all attributes in rule are accounted for.
				{
					if (!HasAttribValue(element.Attributes, a.Name, a.Value, partial))
						hasAllAttributes = false;
				}
				if (hasAllAttributes && HasAttribValue(element.Attributes, rule.AttributeName, rule.AttributeValue, partial))
					return true;
			}
			return false;
		}

		private static bool IsElementInExclusionList(HtmlElement element, List<Rule> Exclusions)
		{
			bool b = false;
			foreach (Rule rule in Exclusions)
				if (IsElementRule(element, rule, false))
					b = true;
			return b;

		}
		private static bool HasAttribValue(HtmlAttributeCollection attribs, string attName, string attValue, bool partial)
		{

			if ((attName == Common.Attribute.WILDCARD) || (attName == string.Empty && attValue == string.Empty && attribs.Count == 0)) return true;

			foreach (HtmlAttribute att in attribs)
			{
				if (att.Name.ToLower() == attName.ToLower())
				{
					if (attValue == Common.Attribute.WILDCARD) return true;
					if (!partial)
					{
						if (att.Value.ToLower() == attValue.ToLower()) return true;
					}
					else
						if (att.Value.ToLower().Contains(attValue.ToLower())) return true;

				}

			}
			return false;
		}

		private static bool HasAttribValue(HtmlAttributeCollection attribs, string attName, string attValue)
		{
			return HasAttribValue(attribs, attName, attValue, false);
		}

	}

	
}
